import os
import time
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from bs4 import BeautifulSoup
import pandas as pd
root_dir = "path/to/the/location/of/this/folder"
download_dir = f"{root_dir}/Downloads/Raw"

# Define the base directory (the directory where the script is located)
base_dir = os.path.dirname(os.path.abspath(__file__))

# Define the relative download directory


# Ensure the download directory exists
os.makedirs(download_dir, exist_ok=True)

# Configure Chrome options to set the download directory
chrome_options = Options()
prefs = {
    'download.default_directory': download_dir,
    'download.prompt_for_download': False,
    'download.directory_upgrade': True,
    'safebrowsing.enabled': True
}
chrome_options.add_experimental_option('prefs', prefs)

# Initialize WebDriver (e.g., Chrome)
#service = Service(r"C:\Users\twk54\Desktop\Programs\Crome_Driver\chrome.exe")  # Ensure the path is correct
driver = webdriver.Chrome(chrome_options)
url_table = f"{root_dir}/Data/Tables/NYSERDA DER Integrated Data System(full).csv"
df = pd.read_csv(url_table)

# List of project URLs
project_urls = df["URL"].to_list()
download_links = []

# Iterate through each project URL
for url in project_urls:
    print("Getting URL:", url)
    driver.get(url)
    time.sleep(2)  # Wait for the page to load

    try:
        # Find and click the "Data" button using its id
        data_button = driver.find_element(By.ID, "download-data-tab")
        print("Found data button, clicking it.")
        data_button.click()
        time.sleep(2)  # Wait for the next button to appear

        # Find and click the "HR" button using its label for attribute
        hr_button = driver.find_element(By.XPATH, "//label[@for='download-data-control-tstep-hour']")
        print("Found HR button, clicking it.")
        hr_button.click()
        time.sleep(2)  # Wait for the download button to appear

        # Find and click the download button using its title attribute
        download_button = driver.find_element(By.XPATH, "//button[@title='Download all performance data from this project in CSV format']")
        print("Found download button, clicking it.")
        download_button.click()
        time.sleep(2)  # Wait for the link to appear

        # Parse the page with BeautifulSoup
        soup = BeautifulSoup(driver.page_source, 'html.parser')

        # Find the download link (modify the selector as needed)
        download_link = soup.find('a', {'class': 'download-link'})['href']
        print("Found download link:", download_link)
        download_links.append(download_link)

    except Exception as e:
        print(f"Failed to process {url}: {e}")
        download_links.append(None)

# Close the WebDriver
driver.quit()

# Save download links to a text file or CSV
df = pd.DataFrame(download_links, columns=['Download Link'])
output_path = os.path.join(download_dir, 'download_links.csv')
print("Saving download links to:", output_path)
df.to_csv(output_path, index=False)